import asyncio

from loguru import logger

from com.arcfox.base.base_spider import ExecuteType
from com.arcfox.manager.redis_task_manager import RedisTaskManager
from com.arcfox.school.parse.enroll_ment_plain_parse import EnrollMentPlainParse
from com.arcfox.school.parse.jzy_major_parse import MajorParser
from com.arcfox.school.parse.province_score_parse import ProvinceScoreParse

from com.arcfox.school.parse.yifen_yiduan_parse import YifenYiduanParser
from com.arcfox.school.processor.jzy.jzy_enroll_ment_plain_processor import EnrollMentPlainProcessor
from com.arcfox.school.processor.jzy.jzy_line_score_processor import LineScoreProcessor
from com.arcfox.school.spider.jzy.jzy_base_spider import JZYBaseSpider
from com.arcfox.util import async_request as requests, util
from com.arcfox.util.util import random_sleep
from com.arcfox.middleware.proxy_zhima_middleware import get_proxy, format_proxy

'''
    一分一段
'''


class ProvinceScoreSpider(JZYBaseSpider):
    def __init__(self):
        super().__init__()
        self.parser = ProvinceScoreParse()
        self.processor = LineScoreProcessor()
        self.task_manager = RedisTaskManager(self.TASK_SCHOOL_PROVINCE_LIST_JZY)

    async def _pull_task(self):
        return await self.task_manager.pull_tasks(100), ExecuteType.FINISH

    async def _crawl_by_task(self, tasks):
        await self.init_session()
        # proxy = await get_proxy()
        formatted_proxy = None
        for task in tasks:
            result_url_list = self.parser.parse_province_score_url_list(self.get_province_map(), task["id"])
            for parse_result in result_url_list:
                try:
                    resp = await requests.get(self.session, url=parse_result["url"], headers=self.get_default_header(),
                                              proxy=formatted_proxy, timeout=3)
                    if resp.code == 200:
                        logger.info("访问：{}成功".format(parse_result["url"]))
                        result_list = self.parser.parse_score_result_list(resp.json(), task["school_name"])
                        # 获取数量
                        await self.processor.store_school_score_province(result_list)

                        # 只获取一次
                        if task["school_name"] == "北京大学":
                            logger.info("爬取省控线")
                            province_list_data = self.parser.parse_province_line_score_result_list(resp.json(),
                                                                                                   self.get_province_map())
                            await self.processor.store_province(province_list_data)
                    if resp.code == -100:
                        proxy = await get_proxy()
                        formatted_proxy = format_proxy(proxy)
                        logger.info("命中风控, 更换 IP")
                        # await self.init_session()
                except asyncio.TimeoutError as e:
                    proxy = await get_proxy()
                    formatted_proxy = format_proxy(proxy)
                    logger.error("命中风控, 休眠10s{}", e)
                    await random_sleep(10)
                    await self.task_manager.add_fail_tasks(task)


if __name__ == "__main__":
    asyncio.run(ProvinceScoreSpider().start_crawl())
